弹力设计篇之"认识故障和弹力设计" - [2026重制版]

# pdb-example.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: my-app-pdb
spec:
  minAvailable: 2  # 至少保持2个可用副本
  # maxUnavailable: 1  # 或者最多允许1个不可用
  selector:
    matchLabels:
      app: my-application
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-application
spec:
  replicas: 3
  selector:
    matchLabels:
      app: my-application
  template:
    metadata:
      labels:
        app: my-application
    spec:
      containers:
      - name: my-app
        image: my-app:v1.0.0
        resources:
          requests:
            cpu: 100m
            memory: 128Mi
          limits:
            cpu: 500m
            memory: 512Mi
        livenessProbe:
          httpGet:
            path: /health/live
            port: 8080
          initialDelaySeconds: 30
          periodSeconds: 10
        readinessProbe:
          httpGet:
            path: /health/ready
            port: 8080
          initialDelaySeconds: 5
          periodSeconds: 5

# hpa-example.yaml
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: my-app-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: my-application
  minReplicas: 2
  maxReplicas: 10
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 70  # CPU利用率达到70%时开始扩容
  - type: Pods
    pods:
      metric:
        name: http_requests_per_second  # 自定义指标
      target:
        type: AverageValue
        averageValue: "1000"  # 每个Pod平均处理1000 QPS
  behavior:
    scaleUp:
      stabilizationWindowSeconds: 60  # 扩容稳定窗口
      policies:
      - type: Percent
        value: 100  # 每次最多扩容100%
        periodSeconds: 15
      - type: Pods
        value: 4  # 每次最多增加4个Pod
        periodSeconds: 15
      selectPolicy: Max  # 选择更激进的策略
    scaleDown:
      stabilizationWindowSeconds: 300  # 缩容稳定窗口（防止抖动）
      policies:
      - type: Percent
        value: 25  # 每次最多缩容25%
        periodSeconds: 15

# vpa-example.yaml
apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
  name: my-app-vpa
spec:
  targetRef:
    apiVersion: "apps/v1"
    kind: Deployment
    name: my-application
  updatePolicy:
    updateMode: "Auto"  # Auto: 自动更新; Off: 仅建议
  resourcePolicy:
    containerPolicies:
    - containerName: my-app
      minAllowed:
        cpu: 50m
        memory: 64Mi
      maxAllowed:
        cpu: 2
        memory: 2Gi
      controlledResources: ["cpu", "memory"]

# istio-destination-rule.yaml
# 目标规则：定义服务的弹力策略
apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: payment-service
spec:
  host: payment-service
  trafficPolicy:
    connectionPool:
      tcp:
        maxConnections: 100  # TCP最大连接数
        connectTimeout: 2s   # 连接超时
      http:
        h2UpgradePolicy: DEFAULT
        http1MaxPendingRequests: 50  # HTTP1最大等待请求数
        http2MaxRequests: 1000       # HTTP2最大请求数
        idleTimeout: 300s            # 空闲超时
        requestTimeout: 10s          # 请求超时
    outlierDetection:  # 异常检测（熔断）
      consecutive5xxErrors: 5       # 连续5个5xx错误
      interval: 30s                 # 检测间隔
      baseEjectionTime: 30s         # 基础驱逐时间
      maxEjectionPercent: 50        # 最大驱逐比例50%
      minHealthPercent: 50          # 最小健康实例比例50%
    tls:
      mode: ISTIO_MUTUAL  # mTLS双向认证
---
# istio-virtual-service.yaml
# 虚拟服务：定义流量管理和重试策略
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: payment-service
spec:
  hosts:
  - payment-service
  http:
  - match:
    - uri:
        prefix: /api/payment
    retries:
      attempts: 3           # 重试次数
      perTryTimeout: 2s     # 每次重试超时
      retryOn: gateway-error,connect-failure,refused-stream  # 重试条件
    timeout: 10s            # 总超时时间
    route:
    - destination:
        host: payment-service
        subset: v1
        port:
          number: 8080
    fault:  # 故障注入（用于混沌工程测试）
      delay:
        percentage:
          value: 0.1  # 0.1%的概率
        fixedDelay: 5s  # 延迟5秒
      abort:
        percentage:
          value: 0.01  # 0.01%的概率
        httpStatus: 500  # 返回500错误

<dependency>
    <groupId>io.github.resilience4j</groupId>
    <artifactId>resilience4j-spring-boot3</artifactId>
    <version>2.1.0</version>
</dependency>
<dependency>
    <groupId>io.github.resilience4j</groupId>
    <artifactId>resilience4j-circuitbreaker</artifactId>
    <version>2.1.0</version>
</dependency>
<dependency>
    <groupId>io.github.resilience4j</groupId>
    <artifactId>resilience4j-ratelimiter</artifactId>
    <version>2.1.0</version>
</dependency>
<dependency>
    <groupId>io.github.resilience4j</groupId>
    <artifactId>resilience4j-retry</artifactId>
    <version>2.1.0</version>
</dependency>
<dependency>
    <groupId>io.github.resilience4j</groupId>
    <artifactId>resilience4j-bulkhead</artifactId>
    <version>2.1.0</version>
</dependency>

resilience4j:
  circuitbreaker:
    instances:
      paymentService:
        registerHealthIndicator: true
        slidingWindowType: COUNT_BASED  # 基于计数
        slidingWindowSize: 20         # 滑动窗口大小
        minimumNumberOfCalls: 5       # 最小调用次数
        failureRateThreshold: 50      # 失败率阈值50%
        waitDurationInOpenState: 30s  # 熔断等待时间
        permittedNumberOfCallsInHalfOpenState: 3  # 半开状态允许调用量
        automaticTransitionFromOpenToHalfOpenEnabled: true
  ratelimiter:
    instances:
      paymentService:
        limitForPeriod: 100           # 每周期允许请求数
        limitRefreshPeriod: 1s        # 刷新周期
        timeoutDuration: 0             # 等待超时
  retry:
    instances:
      paymentService:
        maxAttempts: 3                # 最大重试次数
        waitDuration: 500ms           # 重试间隔
        exponentialBackoffMultiplier: 2  # 指数退避乘数
        retryExceptions:
          - org.springframework.web.client.HttpServerErrorException
        ignoreExceptions:
          - org.springframework.web.client.ClientErrorException
  bulkhead:
    instances:
      paymentService:
        maxConcurrentCalls: 20         # 最大并发数
        maxWaitDuration: 0             # 最大等待时间

@Service
public class PaymentService {

    @Autowired
    private PaymentClient paymentClient;

    @Autowired
    private CircuitBreakerRegistry circuitBreakerRegistry;

    @Autowired
    private RateLimiterRegistry rateLimiterRegistry;

    /**
     * 使用注解方式整合多种弹力模式
     */
    @CircuitBreaker(name = "paymentService", fallbackMethod = "fallbackProcessPayment")
    @RateLimiter(name = "paymentService")
    @Retry(name = "paymentService")
    @Bulkhead(name = "paymentService", type = Bulkhead.Type.THREADPOOL)
    public PaymentResult processPayment(PaymentRequest request) {
        // 业务逻辑
        return paymentClient.charge(request);
    }

    /**
     * 降级方法
     */
    public PaymentResult fallbackProcessPayment(PaymentRequest request, Exception ex) {
        log.warn("Payment service fallback triggered, reason: {}", ex.getMessage());
        // 返回降级结果或走异步队列
        return PaymentResult.pending(request.getOrderId());
    }

    /**
     * 编程式方式（更灵活的控制）
     */
    public PaymentResult processPaymentProgrammatic(PaymentRequest request) {
        // 获取或创建Circuit Breaker
        CircuitBreaker circuitBreaker = circuitBreakerRegistry
            .circuitbreaker("paymentService");

        // 获取或创建Rate Limiter
        RateLimiter rateLimiter = rateLimiterRegistry
            .rateLimiter("paymentService");

        // 使用Supplier装饰器链式组合
        Supplier<PaymentResult> supplier = RateLimiter.decorateSupplier(
            rateLimiter,
            () -> paymentClient.charge(request)
        );

        supplier = CircuitBreaker.decorateSupplier(circuitBreaker, supplier);

        // 执行并处理结果
        try {
            return supplier.get();
        } catch (CallNotPermittedException e) {
            // 熔断打开时的处理
            log.error("Circuit breaker is OPEN for payment service");
            return PaymentResult.rejected("System busy, please try later");
        } catch (RequestNotPermittedException e) {
            // 限流触发时的处理
            log.warn("Rate limit exceeded for payment service");
            throw new TooManyRequestsException("Too many requests");
        }
    }
}

# chaos-mesh-stress-test.yaml
apiVersion: chaos-mesh.io/v1alpha1
kind: StressChaos
metadata:
  name: cpu-stress-test
  namespace: production
spec:
  mode: one  # 随机选择一个Pod
  selector:
    namespaces:
    - production
    labelSelectors:
      app: payment-service
  stressors:
    cpu:
      workers: 4       # CPU压力工作线程数
      load: 80         # CPU负载百分比
      duration: '5m'   # 持续时间
---
apiVersion: chaos-mesh.io/v1alpha1
kind: NetworkChaos
metadata:
  name: network-delay
  namespace: production
spec:
  action: delay  # 网络延迟
  mode: all     # 影响所有匹配Pod
  selector:
    labelSelectors:
      app: inventory-service
  delay:
    latency: "200ms"  # 延迟200毫秒
    correlation: "25" # 相关系数（控制延迟分布的相关性）
    jitter: "50ms"    # 随机抖动50ms
  duration: "3m"     # 持续时间
  scheduler:
    cron: "@every 2h"  # 每2小时执行一次